Retry_dir=Retry
if [ ! -d $Retry_dir ]; then mkdir $Retry_dir; fi

timetag=`date +%Y%m%d%H%M`
check_jobID=-1
nFinished=-1
checkFailedID() {
    pbook -c "show($1)"  > tmp_status
    jobStatus=tmp_status
    nFailed=`less $jobStatus | grep failed | awk '{print $3}'`
    nCancelled=`less $jobStatus | grep cancelled | awk '{print $3}'`
    nFinished=`less $jobStatus | grep finished | awk '{print $3}'`
    if [ -z "${nFinished}" ]; then nFinished=0; fi
    echo "failed: $nFailed; cancelled: $nCancelled; finished: $nFinished"
    if [ -n "${nFailed}" ] || [ -n "${nCancelled}" ]; then
        retrySetID=`less $jobStatus | grep retrySetID | awk '{print $3}'`
        retryID=`less $jobStatus | grep retryID | awk '{print $3}'`
        if [ -n "${retryID}" ] && [ "${retryID}" != "0" ]; then
            checkFailedID $retryID
        elif [ -n "${retrySetID}" ] && [ "${retrySetID}" != "0" ]; then
            checkFailedID $retrySetID
        else 
            check_jobID=$1
        fi
    else 
        check_jobID=-1    
    fi

}

pbook -c "sync()"
 
while read status job ; do
    if [ -z "${status}" ] || [ -z "${job}" ] ; then
        continue
    fi
    while read job2 output Njobs; do
        if [ -z "${job2}" ] || [ -z "${output}" ] || [ -z "${Njobs}" ]; then
            continue
        fi
        if [ "${job2}" == "${job}" ]; then
            break
        fi
    done < history/submitted
    check_jobID=$job
    nFinished=-1
    checkFailedID $job
    echo "failed_jobID: $check_jobID"
    if [ "${check_jobID}" == "-1" ]; then
        echo "Succeeded: $job" >> $Retry_dir/good_job_$timetag
        echo "${job} ${output} ${Njobs}" >> $Retry_dir/completed_job_$timetag
    else 
        echo "Failed: $check_jobID" >> $Retry_dir/failed_job_$timetag
        echo "${output}" >> $Retry_dir/failed_run_$timetag
    fi
done < Retry/failed_job_201301072231




